
#ifndef _UTF8Tokenize_h_
#define _UTF8Tokenize_h_

#include <sstream>
#include <fstream>
#include <string>

#define         UTF8_MASK1BYTE               0x80
#define         UTF8_MASK2BYTES              0xC0
#define         UTF8_MASK3BYTES              0xE0
#define         UTF8_MASK4BYTES              0xF0
#define         UTF8_MASK5BYTES              0xF8
#define         UTF8_MASK6BYTES              0xFC

enum TOKEN_TYPE
{
    TOKEN_EOF   = 0,
    TOKEN_WORD,
    TOKEN_NUMBER,
    TOKEN_CHINESE,
    TOKEN_UNKNOWN
};

class UTF8Token
{
public:
    UTF8Token(::std::string iContent, TOKEN_TYPE iType);
    ~UTF8Token();

    ::std::string   getContent();
    TOKEN_TYPE      getType();

private:
    TOKEN_TYPE      mType;
    ::std::string   mContent;
};

class UTF8Tokenizer
{
public:
	UTF8Tokenizer( ::std::istream& iInputStream );
	~UTF8Tokenizer();
	
	UTF8Token nextToken();

private:
    void skipBlank();

	::std::ostringstream	mStringBuffer;
    ::std::istream&    		mInputStream;
};

#endif

